package cn.edu.fudan.dsm.wikipedia.wordcount.plain;

import java.io.File;
import java.io.FileOutputStream;
import java.util.HashMap;

/**
 * Created by Jiaye Wu on 17-3-24.
 */
public class WordCount {

    private static final HashMap<String, Integer> counts = new HashMap<>();

    public static void main(String[] args) throws Exception {
        System.setProperty("jdk.xml.totalEntitySizeLimit", String.valueOf(0));

        Iterable<Page> pages = new Pages(2000000, "enwiki.xml");
        FileOutputStream fos = new FileOutputStream(new File("/data/enwiki.2000000.xml"));
        for (Page page : pages) {
            fos.write(("<text>" + page.getText() + "</text>").getBytes());
        }
        fos.close();

        pages = new Pages(5000000, "enwiki.xml");
        fos = new FileOutputStream(new File("/data/enwiki.5000000.xml"));
        for (Page page : pages) {
            fos.write(("<text>" + page.getText() + "</text>").getBytes());
        }
        fos.close();

        pages = new Pages(10000000, "enwiki.xml");
        fos = new FileOutputStream(new File("/data/enwiki.10000000.xml"));
        for (Page page : pages) {
            fos.write(("<text>" + page.getText() + "</text>").getBytes());
        }
        fos.close();
    }

    private static void countWord(String word) {
        counts.merge(word, 1, (a, b) -> a + b);
    }
}
